<!DOCTYPE html>
<html lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 5.3.0">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/logo.svg" color="#222">

<link rel="stylesheet" href="/css/main.css">


<link rel="stylesheet" href="/lib/font-awesome/css/font-awesome.min.css">

<script id="hexo-configurations">
    var NexT = window.NexT || {};
    var CONFIG = {"hostname":"movie0125.gitee.io","root":"/","scheme":"Mist","version":"7.8.0","exturl":false,"sidebar":{"position":"left","display":"post","padding":18,"offset":12,"onmobile":false},"copycode":{"enable":false,"show_result":false,"style":null},"back2top":{"enable":true,"sidebar":false,"scrollpercent":false},"bookmark":{"enable":false,"color":"#222","save":"auto"},"fancybox":false,"mediumzoom":false,"lazyload":false,"pangu":false,"comments":{"style":"tabs","active":null,"storage":true,"lazyload":false,"nav":null},"algolia":{"hits":{"per_page":10},"labels":{"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}},"localsearch":{"enable":true,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},"motion":{"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},"path":"search.xml"};
  </script>

  <meta name="description" content="高可用服务pcs源码分析  pcs服务是用于管理 pacemaker 和 corosync 服务，有命令行管理命令和web管理页面，这次主要分析web管理页面相关代码   最近同事在UOS系统中集成PCS高可用服务，在X86和ARM架构上一切正常，但是在龙芯平台上就出现各种奇葩问题，命令行查看时集群状态正常，但是web页面查看集群状态时cluster status，node status等异常。同">
<meta property="og:type" content="article">
<meta property="og:title" content="pcs-in-uos">
<meta property="og:url" content="https://movie0125.gitee.io/2020/10/22/pcs-in-uos/index.html">
<meta property="og:site_name" content="Movie Blog">
<meta property="og:description" content="高可用服务pcs源码分析  pcs服务是用于管理 pacemaker 和 corosync 服务，有命令行管理命令和web管理页面，这次主要分析web管理页面相关代码   最近同事在UOS系统中集成PCS高可用服务，在X86和ARM架构上一切正常，但是在龙芯平台上就出现各种奇葩问题，命令行查看时集群状态正常，但是web页面查看集群状态时cluster status，node status等异常。同">
<meta property="og:locale" content="zh_CN">
<meta property="article:published_time" content="2020-10-22T12:33:55.000Z">
<meta property="article:modified_time" content="2021-02-24T09:09:15.437Z">
<meta property="article:author" content="Movie">
<meta property="article:tag" content="uos">
<meta name="twitter:card" content="summary">

<link rel="canonical" href="https://movie0125.gitee.io/2020/10/22/pcs-in-uos/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome : false,
    isPost : true,
    lang   : 'zh-CN'
  };
</script>

  <title>pcs-in-uos | Movie Blog</title>
  






  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container use-motion">
    <div class="headband"></div>

    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-nav-toggle">
    <div class="toggle" aria-label="切换导航栏">
      <span class="toggle-line toggle-line-first"></span>
      <span class="toggle-line toggle-line-middle"></span>
      <span class="toggle-line toggle-line-last"></span>
    </div>
  </div>

  <div class="site-meta">

    <a href="/" class="brand" rel="start">
      <span class="logo-line-before"><i></i></span>
      <h1 class="site-title">Movie Blog</h1>
      <span class="logo-line-after"><i></i></span>
    </a>
      <p class="site-subtitle" itemprop="description">拥抱开源，快乐分享</p>
  </div>

  <div class="site-nav-right">
    <div class="toggle popup-trigger">
        <i class="fa fa-search fa-fw fa-lg"></i>
    </div>
  </div>
</div>




<nav class="site-nav">
  <ul id="menu" class="menu">
        <li class="menu-item menu-item-home">

    <a href="/" rel="section"><i class="fa fa-fw fa-home"></i>首页</a>

  </li>
        <li class="menu-item menu-item-about">

    <a href="/about/" rel="section"><i class="fa fa-fw fa-user"></i>关于</a>

  </li>
        <li class="menu-item menu-item-tags">

    <a href="/tags/" rel="section"><i class="fa fa-fw fa-tags"></i>标签</a>

  </li>
        <li class="menu-item menu-item-categories">

    <a href="/categories/" rel="section"><i class="fa fa-fw fa-th"></i>分类</a>

  </li>
        <li class="menu-item menu-item-archives">

    <a href="/archives/" rel="section"><i class="fa fa-fw fa-archive"></i>归档</a>

  </li>
      <li class="menu-item menu-item-search">
        <a role="button" class="popup-trigger"><i class="fa fa-search fa-fw"></i>搜索
        </a>
      </li>
  </ul>
</nav>



  <div class="search-pop-overlay">
    <div class="popup search-popup">
        <div class="search-header">
  <span class="search-icon">
    <i class="fa fa-search"></i>
  </span>
  <div class="search-input-container">
    <input autocomplete="off" autocapitalize="off"
           placeholder="搜索..." spellcheck="false"
           type="search" class="search-input">
  </div>
  <span class="popup-btn-close">
    <i class="fa fa-times-circle"></i>
  </span>
</div>
<div id="search-result">
  <div id="no-result">
    <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
  </div>
</div>

    </div>
  </div>

</div>
    </header>

    
  <div class="back-to-top">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>
  <div class="reading-progress-bar"></div>


    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          

          <div class="content post posts-expand">
            

    
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block" lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="https://movie0125.gitee.io/2020/10/22/pcs-in-uos/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="/images/avatar.gif">
      <meta itemprop="name" content="Movie">
      <meta itemprop="description" content="学习，记录，分享，收获">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="Movie Blog">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          pcs-in-uos
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>

              <time title="创建时间：2020-10-22 20:33:55" itemprop="dateCreated datePublished" datetime="2020-10-22T20:33:55+08:00">2020-10-22</time>
            </span>
              <span class="post-meta-item">
                <span class="post-meta-item-icon">
                  <i class="fa fa-calendar-check-o"></i>
                </span>
                <span class="post-meta-item-text">更新于</span>
                <time title="修改时间：2021-02-24 17:09:15" itemprop="dateModified" datetime="2021-02-24T17:09:15+08:00">2021-02-24</time>
              </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90/" itemprop="url" rel="index"><span itemprop="name">源码分析</span></a>
                </span>
            </span>

          
            <span class="post-meta-item" title="阅读次数" id="busuanzi_container_page_pv" style="display: none;">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span id="busuanzi_value_page_pv"></span>
            </span>
  
  <span class="post-meta-item">
    
      <span class="post-meta-item-icon">
        <i class="fa fa-comment-o"></i>
      </span>
      <span class="post-meta-item-text">Valine：</span>
    
    <a title="valine" href="/2020/10/22/pcs-in-uos/#valine-comments" itemprop="discussionUrl">
      <span class="post-comments-count valine-comment-count" data-xid="/2020/10/22/pcs-in-uos/" itemprop="commentCount"></span>
    </a>
  </span>
  
  

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <h1 id="高可用服务pcs源码分析"><a href="#高可用服务pcs源码分析" class="headerlink" title="高可用服务pcs源码分析"></a>高可用服务pcs源码分析</h1><p>  pcs服务是用于管理 pacemaker 和 corosync 服务，有命令行管理命令和web管理页面，这次主要分析web管理页面相关代码</p>
<p>  最近同事在UOS系统中集成PCS高可用服务，在X86和ARM架构上一切正常，但是在龙芯平台上就出现各种奇葩问题，命令行查看时集群状态正常，但是web页面查看集群状态时cluster status，node status等异常。同事求助到我这边，之前页没有使用过pcs，硬着头皮上，折腾了几天，还好找到问题原因，把相关处理过程记录下来，分享给大家。</p>
<a id="more"></a>
<h2 id="目录结构"><a href="#目录结构" class="headerlink" title="目录结构"></a>目录结构</h2><figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">pcs-0.10.1</span><br><span class="line">|- pcs         # 命令行管理，python编码，主要调用 crm_mon 等 pacemaker相关命令来实现</span><br><span class="line">|- pcsd        # web管理页面， ruby编码，通过api相关</span><br><span class="line">   |- public   # ajax等异步调用 js 代码</span><br><span class="line">   |- views    # 前端页面 template 模板</span><br><span class="line">   |- pcsd.rb  # 接受 API 请求，并处理</span><br><span class="line">   |- pcs.rb   # 请求处理函数</span><br><span class="line">   |- remote.rb # 处理 pcsd 转发的请求</span><br><span class="line"></span><br></pre></td></tr></table></figure>
<h2 id="过程分享"><a href="#过程分享" class="headerlink" title="过程分享"></a>过程分享</h2><blockquote>
<p>判断问题范围</p>
</blockquote>
<p>  网上找了一些pcs的架构文档，想知道都有哪些组件，但是在服务器上看只有三个服务：<code>pcsd</code>，<code>pacemaker</code>，<code>corosync</code>。</p>
<ul>
<li><p>通过命令 <code>pcs status</code> 检查服务状态,发现一切正常</p>
</li>
<li><p>通过web页面检查服务状态，发现集群状态各种异常</p>
<p>通过 <code>dpkg -L pcs</code> 检查pcs包都有哪些文件，发现pcs使用python实现命令行工具，使用ruby实现web管理页面。</p>
<p>到这里可以确定出问题的地方是pcs的web管理页面代码</p>
</li>
</ul>
<blockquote>
<p><del>Buff</del> Debug 日志</p>
</blockquote>
<ul>
<li><p>/etc/default/pcsd 打开调试日志</p>
</li>
<li><p>浏览器开启开发者工具</p>
<p>浏览器查看发现 /clusters_overview 请求超时</p>
<p>日志发现如下报错</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">I, [2020-10-20T16:58:02.265 #00006]     INFO -- : Connecting to: https:&#x2F;&#x2F;node1:2224&#x2F;remote&#x2F;check_auth?</span><br><span class="line">I, [2020-10-20T16:58:04.265 #00006]     INFO -- : SRWT Node: node1 Request: cluster_status</span><br><span class="line">I, [2020-10-20T16:58:04.266 #00006]     INFO -- : Connecting to: https:&#x2F;&#x2F;node1:2224&#x2F;remote&#x2F;cluster_status</span><br><span class="line">I, [2020-10-20T16:58:12.266 #00006]     INFO -- : No response from: node1 request: cluster_status, error: operation_timedout</span><br></pre></td></tr></table></figure>
<p>发现是 <code>/remote/cluster_status</code> 和 <code>/remote/status</code> 请求超时,</p>
</li>
</ul>
<blockquote>
<p>查看源码</p>
</blockquote>
<p>  发现请求入口在这里 <code>/usr/share/pcsd/pcsd.rb</code></p>
<figure class="highlight ruby"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 接收 https://xxx:2224/clusters_overview 请求</span></span><br><span class="line"><span class="comment"># 转发给 remote.rb 的  clusters_overview 函数处理</span></span><br><span class="line">get <span class="string">&#x27;/clusters_overview&#x27;</span> <span class="keyword">do</span></span><br><span class="line">  clusters_overview(params, request, getAuthUser())</span><br><span class="line"><span class="keyword">end</span></span><br></pre></td></tr></table></figure>
<p><code>clusters_overview</code> 函数中调用 <code>/remote/cluster_status</code> 请求，timeout 8秒</p>
<figure class="highlight ruby"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">send_request_with_token(</span><br><span class="line">  auth_user, node, <span class="string">&#x27;cluster_status&#x27;</span>, <span class="literal">true</span>, &#123;&#125;, <span class="literal">true</span>, <span class="literal">nil</span>, <span class="number">8</span>)</span><br></pre></td></tr></table></figure>
<p><code>cluster_status_from_nodes</code> 函数中调用 <code>/remote/status</code> 请求，timeout 15秒</p>
<figure class="highlight ruby"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">send_request_with_token(</span><br><span class="line"> auth_user, node, <span class="string">&#x27;status&#x27;</span>, <span class="literal">false</span>,</span><br><span class="line"> &#123;<span class="symbol">:version=&gt;<span class="string">&#x27;2&#x27;</span></span>, <span class="symbol">:operations=&gt;<span class="string">&#x27;1&#x27;</span></span>&#125;, <span class="literal">true</span>, <span class="literal">nil</span>, <span class="number">15</span>)</span><br></pre></td></tr></table></figure>
<p><code>remote.rb</code> 收到请求后，执行函数 <code>node_status</code></p>
<p><code>node_status</code> 函数中判断<code>node</code>是否为本节点，如果不是本节点，转发请求到其他<code>pcsd/remote.rb</code>，如果是本节点，执行<code>pcs.rb</code>中的 <code>get_node_status</code>函数，</p>
<figure class="highlight ruby"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">if</span> params[<span class="symbol">:node</span>] <span class="keyword">and</span> params[<span class="symbol">:node</span>] != <span class="string">&#x27;&#x27;</span> <span class="keyword">and</span> params[<span class="symbol">:node</span>] !=</span><br><span class="line">    $cur_node_name <span class="keyword">and</span> !params[<span class="symbol">:redirected</span>]</span><br><span class="line">    <span class="keyword">return</span> send_request_with_token(xxx,<span class="string">&#x27;status?redirected=1&#x27;</span>，xxx）</span><br><span class="line">……</span><br><span class="line">  cib_dom = get_cib_dom(auth_user)</span><br><span class="line">  crm_dom = get_crm_mon_dom(auth_user)</span><br><span class="line"></span><br><span class="line">  status = get_node_status(auth_user, cib_dom)</span><br><span class="line">……</span><br></pre></td></tr></table></figure>
<p><code>get_node_status</code> 函数调用 <code>pcs status nodes both</code> 命令查询节点状态。</p>
<blockquote>
<p>增加 Debug 日志</p>
</blockquote>
<p>  从整个源码流程来看，有点绕，这是因为服务每拆分，看起来有点乱。<br>  为了更好明白相关代码逻辑，增加了一些变量打印的debug日志</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">D, [2020-10-20T17:21:52.456 #00179]    DEBUG -- : return_code is operation_timedout</span><br><span class="line">D, [2020-10-20T17:21:52.456 #00179]    DEBUG -- : timeout_ms is 8000</span><br><span class="line">D, [2020-10-20T17:21:52.457 #00179]    DEBUG -- : cookie is CIB_user&#x3D;hacluster; path&#x3D;;CIB_user_groups&#x3D;aGFjbGllbnQ%3D; path&#x3D;;token&#x3D;fea435a6-7d53-4c5d-b966-05b138f75629; path&#x3D;</span><br><span class="line">D, [2020-10-20T17:21:52.457 #00179]    DEBUG -- : postfields is </span><br><span class="line">D, [2020-10-20T17:21:52.457 #00179]    DEBUG -- : httpget is true</span><br></pre></td></tr></table></figure>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre></td><td class="code"><pre><span class="line">D, [2020-10-21T12:20:08.440 #00708]    DEBUG -- : permission check action&#x3D;read username&#x3D;hacluster groups&#x3D;haclient</span><br><span class="line">D, [2020-10-21T12:20:08.440 #00708]    DEBUG -- : permission granted for superuser</span><br><span class="line">I, [2020-10-21T12:20:08.440 #00708]     INFO -- : Running: &#x2F;usr&#x2F;sbin&#x2F;pcs status nodes both</span><br><span class="line">I, [2020-10-21T12:20:08.440 #00708]     INFO -- : CIB USER: hacluster, groups: </span><br><span class="line">D, [2020-10-21T12:20:09.441 #00708]    DEBUG -- : [&#39;Corosync Nodes:\n&#39;, &#39; Online: node1\n&#39;, &#39; Offline:\n&#39;, &#39;Pacemaker Nodes:\n&#39;, &#39; Online: node1\n&#39;, &#39; Standby:\n&#39;, &#39; Maintenance:\n&#39;, &#39; Offline:\n&#39;, &#39;Pacemaker Remote Nodes:\n&#39;, &#39; Online:\n&#39;, &#39; Standby:\n&#39;, &#39; Maintenance:\n&#39;, &#39; Offline:\n&#39;]</span><br><span class="line">D, [2020-10-21T12:20:09.441 #00708]    DEBUG -- : []</span><br><span class="line">D, [2020-10-21T12:20:09.441 #00708]    DEBUG -- : Duration: 0.775261534s</span><br><span class="line">I, [2020-10-21T12:20:09.441 #00708]     INFO -- : Return Value: 0</span><br><span class="line">I, [2020-10-21T12:20:09.442 #00708]     INFO -- : SRWT Node: node1 Request: status</span><br><span class="line">I, [2020-10-21T12:20:09.442 #00708]     INFO -- : Connecting to: https:&#x2F;&#x2F;node1:2224&#x2F;remote&#x2F;status?version&#x3D;2&amp;operations&#x3D;1</span><br><span class="line">I, [2020-10-21T12:20:24.442 #00708]     INFO -- : No response from: node1 request: status, error: operation_timedout</span><br><span class="line">D, [2020-10-21T12:20:24.442 #00708]    DEBUG -- : &#x3D;&#x3D;&#x3D;&#x3D;&#x3D;mydebug&#x3D;&#x3D;&#x3D;&#x3D;cluster_name&#x3D;&#x3D;&#x3D;&#x3D;mycluster</span><br><span class="line">D, [2020-10-21T12:20:24.442 #00708]    DEBUG -- : &#x3D;&#x3D;&#x3D;&#x3D;&#x3D;mybudeg&#x3D;&#x3D;&#x3D;&#x3D;cluster_nodes&#x3D;&#x3D;&#x3D;&#x3D;&#123;:cluster_name&#x3D;&gt;&quot;mycluster&quot;, :error_list&#x3D;&gt;[], :warning_list&#x3D;&gt;[], :quorate&#x3D;&gt;false, :status&#x3D;&gt;&quot;error&quot;, :node_list&#x3D;&gt;[&#123;:name&#x3D;&gt;&quot;node1&quot;, :status&#x3D;&gt;&quot;unknown&quot;, :warning_list&#x3D;&gt;[], :error_list&#x3D;&gt;[]&#125;], :resource_list&#x3D;&gt;[], :available_features&#x3D;&gt;[], :pcsd_capabilities&#x3D;&gt;[]&#125;</span><br><span class="line">D, [2020-10-21T12:20:24.442 #00708]    DEBUG -- : &#x3D;&#x3D;&#x3D;&#x3D;&#x3D;mybudeg&#x3D;&#x3D;&#x3D;&#x3D;status&#x3D;&#x3D;&#x3D;&#x3D;&#123;:cluster_name&#x3D;&gt;&quot;mycluster&quot;, :error_list&#x3D;&gt;[], :warning_list&#x3D;&gt;[], :quorate&#x3D;&gt;false, :status&#x3D;&gt;&quot;error&quot;, :node_list&#x3D;&gt;[&#123;:name&#x3D;&gt;&quot;node1&quot;, :status&#x3D;&gt;&quot;unknown&quot;, :warning_list&#x3D;&gt;[], :error_list&#x3D;&gt;[]&#125;], :resource_list&#x3D;&gt;[], :available_features&#x3D;&gt;[], :pcsd_capabilities&#x3D;&gt;[]&#125;</span><br><span class="line">I, [2020-10-21T12:20:24.443 #00000]     INFO -- : 200 POST &#x2F;remote&#x2F;cluster_status (10.20.13.106) 17311.71ms</span><br><span class="line"></span><br></pre></td></tr></table></figure>
<p>查看 cluster_status相关请求处理日志，所以请求是能正常返回 200的，只是超时返回，查看超时设置，发现 <code>send_request_with_token</code> 函数中最后一个参数（其实是倒数第二个，最后一个一般是默认值）为 timeout，调整timeout值后重试，<br><code>/remote/status</code> 和 <code>/remote/cluster_status</code> 请求正常。</p>
<figure class="highlight plain"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">#00908号线程  12:45:52  发送请求  &#x2F;remote&#x2F;status?xxx</span><br><span class="line">#00908号线程  12.46:07  返回超时失败，中间15秒</span><br><span class="line"></span><br><span class="line">#00909号线程  12:46:10  才返回请求成功，一共经过18秒</span><br></pre></td></tr></table></figure>
<p>但是 <code>/clusters_overview</code> 请求还是异常,调整函数 <code>clusters_overview</code> 中代码,线程总timeout，但是问题还是存在。</p>
<figure class="highlight ruby"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">begin</span></span><br><span class="line">  Timeout::timeout(<span class="number">18</span>) &#123;</span><br><span class="line">    threads.each &#123; <span class="params">|t|</span> t.join &#125;</span><br><span class="line">  &#125;</span><br><span class="line"><span class="keyword">rescue</span> Timeout::Error</span><br><span class="line">  threads.each &#123; <span class="params">|t|</span> t.exit &#125;</span><br><span class="line"><span class="keyword">end</span></span><br></pre></td></tr></table></figure>
<p>观察浏览器发出 <code>clusters_overview</code> 请求超过20秒，请求就会强制失败。怀疑上游入口还有超时设置。整个项目搜索<code>timeout</code>,<code>20</code>,<code>clusters_overview</code> 等关键字，东西太多，无法有效分析。</p>
<p>观察浏览器发出<code>clusters_overview</code>请求，页面刷新后，页面数据需要一段时间才会刷新，说明有异步任务，异步任务一般都是js实现，pcs中是ajax的异步代码，</p>
<p>重点搜索js代码中关于 <code>clusters_overview</code> 请求的代码，最后在<code>nodes-ember.js</code> 中发现如下代码,修改超时时间，问题解决</p>
<figure class="highlight ruby"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line">Ember.debug(<span class="string">&quot;Empty Cluster Name&quot;</span>);</span><br><span class="line">  ajax_wrapper(&#123;</span><br><span class="line">    <span class="symbol">url:</span> <span class="string">&quot;/clusters_overview&quot;</span>,</span><br><span class="line">    <span class="symbol">dataType:</span> <span class="string">&quot;json&quot;</span>,</span><br><span class="line">    <span class="symbol">timeout:</span> <span class="number">20000</span>,</span><br><span class="line">    <span class="symbol">success:</span> function(data) &#123;</span><br><span class="line">      Pcs.clusterController.update(data);</span><br><span class="line">      <span class="keyword">if</span> (Pcs.clusterController.get(<span class="string">&#x27;cur_cluster&#x27;</span>)) &#123;</span><br><span class="line">         Pcs.clusterController.update_cur_cluster(Pcs.clusterController.get(<span class="string">&#x27;cur_cluster&#x27;</span>).get(<span class="string">&#x27;name&#x27;</span>));</span><br><span class="line">      &#125;</span><br><span class="line">      <span class="keyword">if</span> (data[<span class="string">&quot;not_current_data&quot;</span>]) &#123;</span><br><span class="line">         <span class="keyword">self</span>.update();</span><br><span class="line">      &#125;</span><br><span class="line">      hide_loading_screen();</span><br><span class="line">    &#125;,</span><br></pre></td></tr></table></figure>

<h2 id="流程图"><a href="#流程图" class="headerlink" title="流程图"></a>流程图</h2><pre class="mermaid">sequenceDiagram
participant pcsd.rb
    pcsd.rb->>remote.rb: clusters_overview
    Note right of pcsd.rb: ajax timeout 20

    remote.rb->>pcs_setting.json: get node list from file

    loop thread of each nodes timeout 8
    remote.rb-->>remote.rb: xxx:2224/remote/cluster_status
    end 


    remote.rb->>pcs.rb: 收到请求后执行函数 cluster_status_from_nodes


    pcs.rb-->>remote.rb: xxx:2224/remote/status
    Note right of remote.rb: timeout 15

    remote.rb->>remote.rb: node_status
    remote.rb->>pcs.rb: get_node_status

    pcs.rb->pcs.rb: run cmd "pcs status node both"

    Note right of remote.rb: wait for request /remote/cluster_status  timeout 18</pre>

<h2 id="问题总结"><a href="#问题总结" class="headerlink" title="问题总结"></a>问题总结</h2><ul>
<li><p>必现的问题最好定位，只要找到触发条件，触发条件越简单，复现越快，定位就越简单</p>
</li>
<li><p>必要的调试方法，优先级 debug日志 &gt; 源码 &gt; gdb </p>
</li>
<li><p>python, ruby, shell 等脚本型语言一般不需要 gdb</p>
</li>
<li><p>脚本型语言需要先找到触发方式，找到出问题的函数，增加相关 debug日志，比如pcs源码中增加 $logger.debug “xxxx”</p>
</li>
<li><p>浏览器可以开启开发者模式，观察请求报文，响应报文，请求时长等等非常有帮助</p>
</li>
<li><p>超时问题需要细心观察和判断，超时点出现在哪里，这个案例中，看起来是外面的请求超时，但是实际上存在多层调用关系，每层调用都超时了</p>
</li>
<li><p><strong>开源项目很多，如果不是有相关经验，单独查看源码或者日志，都会觉得很乱，很难入门，直接上手看代码，是很难掌握的，需要结合问题、日志和gdb调试等手段来综合分析源代码逻辑，这样才能更快的进入状态。</strong>（划重点，这条很重要）</p>
</li>
</ul>

    </div>

    
    
    

      <footer class="post-footer">
          <div class="post-tags">
              <a href="/tags/uos/" rel="tag"># uos</a>
          </div>

        


        
    <div class="post-nav">
      <div class="post-nav-item"></div>
      <div class="post-nav-item">
    <a href="/2020/10/26/X11vnc-in-uos/" rel="next" title="X11vnc-in-uos">
      X11vnc-in-uos <i class="fa fa-chevron-right"></i>
    </a></div>
    </div>
      </footer>
    
  </article>
  
  
  



          </div>
          
    <div class="comments" id="valine-comments"></div>

<script>
  window.addEventListener('tabs:register', () => {
    let { activeClass } = CONFIG.comments;
    if (CONFIG.comments.storage) {
      activeClass = localStorage.getItem('comments_active') || activeClass;
    }
    if (activeClass) {
      let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
      if (activeTab) {
        activeTab.click();
      }
    }
  });
  if (CONFIG.comments.storage) {
    window.addEventListener('tabs:click', event => {
      if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
      let commentClass = event.target.classList[1];
      localStorage.setItem('comments_active', commentClass);
    });
  }
</script>

        </div>
          
  
  <div class="toggle sidebar-toggle">
    <span class="toggle-line toggle-line-first"></span>
    <span class="toggle-line toggle-line-middle"></span>
    <span class="toggle-line toggle-line-last"></span>
  </div>

  <aside class="sidebar">
    <div class="sidebar-inner">

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          文章目录
        </li>
        <li class="sidebar-nav-overview">
          站点概览
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
          <div class="post-toc motion-element"><ol class="nav"><li class="nav-item nav-level-1"><a class="nav-link" href="#%E9%AB%98%E5%8F%AF%E7%94%A8%E6%9C%8D%E5%8A%A1pcs%E6%BA%90%E7%A0%81%E5%88%86%E6%9E%90"><span class="nav-number">1.</span> <span class="nav-text">高可用服务pcs源码分析</span></a><ol class="nav-child"><li class="nav-item nav-level-2"><a class="nav-link" href="#%E7%9B%AE%E5%BD%95%E7%BB%93%E6%9E%84"><span class="nav-number">1.1.</span> <span class="nav-text">目录结构</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E8%BF%87%E7%A8%8B%E5%88%86%E4%BA%AB"><span class="nav-number">1.2.</span> <span class="nav-text">过程分享</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E6%B5%81%E7%A8%8B%E5%9B%BE"><span class="nav-number">1.3.</span> <span class="nav-text">流程图</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#%E9%97%AE%E9%A2%98%E6%80%BB%E7%BB%93"><span class="nav-number">1.4.</span> <span class="nav-text">问题总结</span></a></li></ol></li></ol></div>
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <p class="site-author-name" itemprop="name">Movie</p>
  <div class="site-description" itemprop="description">学习，记录，分享，收获</div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/archives/">
        
          <span class="site-state-item-count">14</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/categories/">
          
        <span class="site-state-item-count">3</span>
        <span class="site-state-item-name">分类</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/tags/">
          
        <span class="site-state-item-count">7</span>
        <span class="site-state-item-name">标签</span></a>
      </div>
  </nav>
</div>



      </div>

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer class="footer">
      <div class="footer-inner">
        

        

<div class="copyright">
  
  &copy; 
  <span itemprop="copyrightYear">2022</span>
  <span class="with-love">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Movie</span>
</div>

        
<div class="busuanzi-count">
  <script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
    <span class="post-meta-item" id="busuanzi_container_site_uv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-user"></i>
      </span>
      <span class="site-uv" title="总访客量">
        <span id="busuanzi_value_site_uv"></span>
      </span>
    </span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item" id="busuanzi_container_site_pv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-eye"></i>
      </span>
      <span class="site-pv" title="总访问量">
        <span id="busuanzi_value_site_pv"></span>
      </span>
    </span>
</div>








      </div>
    </footer>
  </div>

  
  <script src="/lib/anime.min.js"></script>
  <script src="/lib/velocity/velocity.min.js"></script>
  <script src="/lib/velocity/velocity.ui.min.js"></script>

<script src="/js/utils.js"></script>

<script src="/js/motion.js"></script>


<script src="/js/schemes/muse.js"></script>


<script src="/js/next-boot.js"></script>




  




  
<script src="/js/local-search.js"></script>











<script>
if (document.querySelectorAll('pre.mermaid').length) {
  NexT.utils.getScript('//cdn.jsdelivr.net/npm/mermaid@8/dist/mermaid.min.js', () => {
    mermaid.initialize({
      theme    : 'forest',
      logLevel : 3,
      flowchart: { curve     : 'linear' },
      gantt    : { axisFormat: '%m/%d/%Y' },
      sequence : { actorMargin: 50 }
    });
  }, window.mermaid);
}
</script>


  

  


<script>
NexT.utils.loadComments(document.querySelector('#valine-comments'), () => {
  NexT.utils.getScript('//unpkg.com/valine/dist/Valine.min.js', () => {
    var GUEST = ['nick', 'mail', 'link'];
    var guest = 'nick,mail,link';
    guest = guest.split(',').filter(item => {
      return GUEST.includes(item);
    });
    new Valine({
      el         : '#valine-comments',
      verify     : false,
      notify     : false,
      appId      : 'bx3R5BB7VYCe0Yu5f4PF2Ako-gzGzoHsz',
      appKey     : '24eOkgM1v8hYTlnAO3C2eG3n',
      placeholder: "欢迎留言评论",
      avatar     : 'mm',
      meta       : guest,
      pageSize   : '10' || 10,
      visitor    : false,
      lang       : 'zh-cn' || 'zh-cn',
      path       : location.pathname,
      recordIP   : false,
      serverURLs : ''
    });
  }, window.Valine);
});
</script>

</body>
</html>
